{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 这个不知道在爬啥 没有文章信息 应该是其他相关的链接吗"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Title: 探讨人才培养新模式　华南理工出访新西兰、澳大利亚高校\n",
      "1. : None\n",
      "2. : /main.htm\n",
      "3. 登录: https://newslogin.scut.edu.cn\n",
      "4. 校报: http://xb.scut.edu.cn/\n",
      "5. 微信: https://mp.weixin.qq.com/profile?src=3×tamp=1524195183&ver=1&signature=Nfm8MDXY4GIE-rN3JzZku1FlYrXgW-Jpa1DJS8vK3J*q-aa9Rhio6EcRaOFvkTrJ7Oo4gHONglpNH2cghyeg4A==\n",
      "6. 微博: https://weibo.com/scut1117?refer_flag=1001030201_\n",
      "7. 华工主页: http://www.scut.edu.cn\n",
      "8. : None\n",
      "9. 校园新闻: /41/list.htm\n",
      "10. 学院风采: /55/list.htm\n",
      "11. 媒体华园: /44/list.htm\n",
      "12. 追梦人: /zmr/list.htm\n",
      "13. 新视点: /73/list.htm\n",
      "14. 记者眼: /59/list.htm\n",
      "15. 理论学习: /107/list.htm\n",
      "16. 专题热点: /136/list.htm\n",
      "17. 精彩视频: /137/list.htm\n",
      "18. 双百行动: /sbxd/list.htm\n",
      "19. 学习贯彻习近平新时代中国特色社会主义思想主题...: https://www2.scut.edu.cn/xxjy2023/\n",
      "20. 学习宣传贯彻党的二十大精神专题网站: https://www2.scut.edu.cn/ddesd/\n",
      "21. 党史学习教育专题网站: http://www2.scut.edu.cn/dsxx/\n",
      "22. 第六届中国国际“互联网+”大学生创新创业大赛: http://www2.scut.edu.cn/internetplus2020/\n",
      "23. 中共华南理工大学第十七次代表大会专题网站: http://www2.scut.edu.cn/17dangdaihui/\n",
      "24. 中国共产党纪律处分条例: /2024/0426/c107a48901/page.htm\n",
      "25. 习近平在视察陆军军医大学时强调 面向战场面向...: /2024/0426/c107a48897/page.htm\n",
      "26. 习近平在重庆考察时强调 进一步全面深化改革开...: /2024/0426/c107a48896/page.htm\n",
      "27. 中共中央办公厅印发《通知》 在全党开展党纪学...: /2024/0426/c107a48898/page.htm\n",
      "28. 习近平在湖南考察时强调 坚持改革创新求真务实 ...: /2024/0426/c107a48895/page.htm\n",
      "29. 倪阳：矢志三十六载 匠筑理想之城: /2024/0603/c161a49206/page.htm\n",
      "30. 陈永琪：“打印”出精彩的3D人生: /2024/0318/c161a48601/page.htm\n",
      "31. 李靖豪：让“华工智慧”闪耀6G星空: /2024/0129/c161a48407/page.htm\n",
      "32. 林伊霞：国风传播 文化创业 她用创意实干点亮华...: /2023/1229/c161a48226/page.htm\n",
      "33. 如果你要提华南理工，就不能只提华南理工: https://mp.weixin.qq.com/s/MrNxRLtGLeK8R0wSXxd4Zw\n",
      "34. 华园版课本封面，适配度拉满！: https://mp.weixin.qq.com/s/RSlZA_n-m6pCiB5hh1ljmQ\n",
      "35. 看完这篇就睡吧！: https://mp.weixin.qq.com/s/hJ3eC8t1WxDRUF9jmSAgBQ\n",
      "36. 春天在哪里？在华工人的眼睛里！: https://mp.weixin.qq.com/s/ehsx08rfu3SgNEeLR2siZw\n",
      "37. 聚焦！全国两会中的华南理工好声音！: https://mp.weixin.qq.com/s/tn9vto3YIS1ueXJAHxAdcQ\n",
      "38. 实用调适指南，助鲤工仔新学期“龙抬头”！: https://mp.weixin.qq.com/s/-_jnlK6VVfDklN9ohcQLCw\n",
      "39. 首页: /main.htm\n",
      "40. 校园新闻: /41/list.htm\n",
      "41. : javascript:void(0)\n",
      "42. : javascript:void(0)\n",
      "43. : javascript:void(0)\n",
      "44. A+: javascript:void(0)\n",
      "45. A-: javascript:void(0)\n",
      "46. 夜晚模式: javascript:void(0)\n",
      "47. 粤ICP备05084312号: https://beian.miit.gov.cn\n",
      "48. 返回: javascript:;\n",
      "49. 原图: None\n"
     ]
    }
   ],
   "source": [
    "import requests\n",
    "from bs4 import BeautifulSoup\n",
    "import chardet\n",
    "\n",
    "def scrape_webpage(url):\n",
    "    try:\n",
    "        # 获取网页内容\n",
    "        response = requests.get(url)\n",
    "        response.raise_for_status()\n",
    "\n",
    "        # 自动检测编码\n",
    "        encoding = chardet.detect(response.content)['encoding']\n",
    "        response.encoding = encoding\n",
    "\n",
    "        # 解析网页内容\n",
    "        page_content = response.text\n",
    "        soup = BeautifulSoup(page_content, 'html.parser')\n",
    "\n",
    "        # 获取标题\n",
    "        title = soup.title.string if soup.title else \"No title found\"\n",
    "        print(f\"Title: {title}\")\n",
    "\n",
    "        # 获取所有链接\n",
    "        links = soup.find_all('a')\n",
    "        for index, link in enumerate(links, start=1):\n",
    "            href = link.get('href')\n",
    "            text = link.get_text(strip=True)\n",
    "            print(f\"{index}. {text}: {href}\")\n",
    "\n",
    "    except requests.exceptions.RequestException as e:\n",
    "        print(f\"Error: {e}\")\n",
    "\n",
    "# 使用爬虫抓取网页\n",
    "url = 'http://news.scut.edu.cn/2024/0603/c41a49210/page.htm'  # 替换为你想要抓取的网页\n",
    "scrape_webpage(url)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 这个可以爬到文章信息"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "网页标题：\n",
      " 中山大学保继刚教授：新时期城市旅游发展战略思考\n",
      "文章内容：\n",
      " 报告题目：\n",
      "新时期城市旅游发展战略思考\n",
      "报 告 人：\n",
      "保继刚（中山大学教授）\n",
      "报告时间：\n",
      "2024\n",
      "年\n",
      "6\n",
      "月\n",
      "18\n",
      "日（星期二）\n",
      "15:00\n",
      "—\n",
      "17:30\n",
      "报告地点：\n",
      "华南理工大学（大学城校区）\n",
      "B10\n",
      "北座一楼学术报告厅\n",
      "主办单位：\n",
      "华南理工大学旅游管理系，广东省旅游控股集团有限公司，广东省旅游协会\n",
      "报告人简介：\n",
      "中山大学旅游学院创院院长（\n",
      "2004\n",
      "—\n",
      "2018\n",
      "）、博士、教授，国家级人才计划入选者，现任中山大学经济与管理学部副主任、旅游发展与规划研究中心主任，联合国世界旅游组织旅游可持续发展观测点管理与监测中心主任，中国旅游协会旅游教育分会会长，教育部旅游管理类专业教学指导委员会副主任委员，\n",
      "2023\n",
      "年获联合国旅游组织第\n",
      "16\n",
      "届尤利西斯奖，\n",
      "2024\n",
      "年获全国五一劳动奖章。发表论文\n",
      "300\n",
      "余篇，出版著作\n",
      "20\n",
      "余部；主持国家自然科学和社会科学艺术类重大重点、面上、青年基金\n",
      "8\n",
      "项；主持广东省、湖北省、湖南省省级旅游发展规划，主持桂林市、苏州市、黄山市、张家界市、西双版纳、珠海市等城市旅游发展规划，主持新疆喀纳斯、大香格里拉等旅游区规划；倡导并主持推动了“阿者科计划”。\n",
      "中山大学保继刚教授：新时期城市旅游发展战略思考(1).docx\n"
     ]
    }
   ],
   "source": [
    "import requests\n",
    "from bs4 import BeautifulSoup\n",
    "import chardet\n",
    "\n",
    "def scrape_notification_content(url):\n",
    "    try:\n",
    "        # 请求网页内容\n",
    "        response = requests.get(url)\n",
    "        response.raise_for_status()\n",
    "\n",
    "        # 自动检测网页编码\n",
    "        encoding = chardet.detect(response.content)['encoding']\n",
    "        response.encoding = encoding\n",
    "\n",
    "        # 解析网页内容\n",
    "        soup = BeautifulSoup(response.text, 'html.parser')\n",
    "        \n",
    "        # 提取网页标题\n",
    "        title = soup.find('title').get_text() if soup.find('title') else '无标题'\n",
    "        print(\"网页标题：\\n\", title)\n",
    "\n",
    "        # 找到文章正文的部分\n",
    "        article = soup.find('div', class_='wp_articlecontent')\n",
    "        if article:\n",
    "            # 提取正文文本\n",
    "            content = article.get_text(separator='\\n', strip=True)\n",
    "            print(\"文章内容：\\n\", content)\n",
    "        else:\n",
    "            print(\"未找到文章内容。\")\n",
    "\n",
    "    except requests.exceptions.RequestException as e:\n",
    "        print(f\"请求错误：{e}\")\n",
    "\n",
    "# 使用爬虫抓取特定页面\n",
    "# 0615新闻\n",
    "# url = 'http://news.scut.edu.cn/2024/0612/c41a49266/page.htm'\n",
    "# url = 'http://news.scut.edu.cn/2024/0611/c41a49262/page.htm'\n",
    "# url = 'http://news.scut.edu.cn/2024/0615/c41a49292/page.htm'\n",
    "# url = 'http://news.scut.edu.cn/2024/0611/c41a49262/page.htm'\n",
    "# 0616新闻\n",
    "url = 'https://www.scut.edu.cn/new/2024/0617/c76a25110/page.htm'\n",
    "\n",
    "scrape_notification_content(url)\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 提取到的内容和标题 保存到路径下的txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "网页标题：\n",
      " 华南理工大学2024年纪检监察干部专题培训班结业\n",
      "文章内容：\n",
      " 6月12日，华南理工大学2024年党纪学习教育纪检监察干部专题培训班集中开展第四次专题学习并举行结业仪式。学校专职纪检干部、各二级纪委书记、纪委委员以及广州国际校区各学院负责监督工作人员共约140人参加培训暨结业仪式。\n",
      "会议现场\n",
      "会上，学校财务处、教务处、党委教师工作部等3个职能部门负责同志作专题辅导，重点围绕财会监督、师德师风、职称评审、转专业和保研等工作对廉洁风险点进行逐一梳理，讲解风险防控措施及可能带来的后果和影响，为二级纪委开展日常监督工作、加强廉洁风险识别与防控提供具体指导。\n",
      "结业式上，学校纪委副书记、纪委办公室主要负责人对本次培训班作总结，希望各二级纪委书记和委员进一步提高政治站位、增强使命担当、提升履职能力，着力解决“不愿监督、不敢监督、不会监督”问题，扎实做好本单位纪检监督、廉洁教育等工作。\n",
      "本次专题培训班重点聚焦《中国共产党纪律处分条例》，以专家解读、案例剖析、实地研学、专题辅导等方式，深入解读重点内容，着力提高解决实际问题能力。参训学员纷纷表示将把学习培训的成果总结归纳好、消化吸收好、实践运用好，努力补齐自身存在的短板弱项，助力学校纪检监察工作高质量发展。\n",
      "内容已保存到 /home/pci/work/gmx/cby_SCUT/chat_cby/docs11111/cleaned_txt/华南理工大学2024年纪检监察干部专题培训班结业.txt\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import re\n",
    "import requests\n",
    "from bs4 import BeautifulSoup\n",
    "import chardet\n",
    "\n",
    "def clean_filename(title):\n",
    "    # 移除文件名中的非法字符\n",
    "    return re.sub(r'[\\/:*?\"<>|]', '', title)\n",
    "\n",
    "def scrape_notification_content(url, output_dir):\n",
    "    try:\n",
    "        # 请求网页内容\n",
    "        response = requests.get(url)\n",
    "        response.raise_for_status()\n",
    "\n",
    "        # 自动检测网页编码\n",
    "        encoding = chardet.detect(response.content)['encoding']\n",
    "        response.encoding = encoding\n",
    "\n",
    "        \n",
    "        \n",
    "        # 解析网页内容\n",
    "        soup = BeautifulSoup(response.text, 'html.parser')\n",
    "\n",
    "        # 提取网页标题\n",
    "        title = soup.find('title').get_text() if soup.find('title') else '无标题'\n",
    "        \n",
    "        # 清理标题以用作文件名\n",
    "        clean_title = clean_filename(title)\n",
    "\n",
    "        # 找到文章正文的部分\n",
    "        article = soup.find('div', class_='wp_articlecontent')\n",
    "        if article:\n",
    "            # 提取正文文本\n",
    "            content = article.get_text(separator='\\n', strip=True)\n",
    "        else:\n",
    "            content = \"未找到文章内容。\"\n",
    "\n",
    "        # 打印文章标题和内容\n",
    "        print(\"网页标题：\\n\", title)\n",
    "        print(\"文章内容：\\n\", content)\n",
    "\n",
    "        # 使用清理后的标题作为文件名\n",
    "        filename = os.path.join(output_dir, f\"{clean_title}.txt\")\n",
    "        with open(filename, 'w', encoding='utf-8') as file:\n",
    "            file.write(f\"标题：\\n{title}\\n\\n\")\n",
    "            file.write(f\"内容：\\n{content}\\n\")\n",
    "\n",
    "        print(f\"内容已保存到 {filename}\")\n",
    "\n",
    "    except requests.exceptions.RequestException as e:\n",
    "        print(f\"请求错误：{e}\")\n",
    "    return content\n",
    "\n",
    "# 使用爬虫抓取特定页面，并保存到指定目录\n",
    "# url = 'https://news.scut.edu.cn/2024/0615/c41a49292/page.htm'  # 替换为你想要爬取的 URL\n",
    "# url = 'https://www.scut.edu.cn/new/2024/0617/c76a25109/page.htm'\n",
    "# url = 'https://www.scut.edu.cn/new/2024/0617/c76a25108/page.htm'\n",
    "url = 'http://news.scut.edu.cn/2024/0614/c41a49284/page.htm'\n",
    "\n",
    "output_directory = '/home/pci/work/gmx/cby_SCUT/chat_cby/docs11111/cleaned_txt'  # 替换为你要保存文件的目录\n",
    "content = scrape_notification_content(url, output_directory)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "6月12日，华南理工大学2024年党纪学习教育纪检监察干部专题培训班集中开展第四次专题学习并举行结业仪式。学校专职纪检干部、各二级纪委书记、纪委委员以及广州国际校区各学院负责监督工作人员共约140人参加培训暨结业仪式。\n",
      "会议现场\n",
      "会上，学校财务处、教务处、党委教师工作部等3个职能部门负责同志作专题辅导，重点围绕财会监督、师德师风、职称评审、转专业和保研等工作对廉洁风险点进行逐一梳理，讲解风险防控措施及可能带来的后果和影响，为二级纪委开展日常监督工作、加强廉洁风险识别与防控提供具体指导。\n",
      "结业式上，学校纪委副书记、纪委办公室主要负责人对本次培训班作总结，希望各二级纪委书记和委员进一步提高政治站位、增强使命担当、提升履职能力，着力解决“不愿监督、不敢监督、不会监督”问题，扎实做好本单位纪检监督、廉洁教育等工作。\n",
      "本次专题培训班重点聚焦《中国共产党纪律处分条例》，以专家解读、案例剖析、实地研学、专题辅导等方式，深入解读重点内容，着力提高解决实际问题能力。参训学员纷纷表示将把学习培训的成果总结归纳好、消化吸收好、实践运用好，努力补齐自身存在的短板弱项，助力学校纪检监察工作高质量发展。\n"
     ]
    }
   ],
   "source": [
    "print(content)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import pdfplumber\n",
    "# import re\n",
    "\n",
    "# def clean_extracted_text(text):\n",
    "#     \"\"\"Clean and preprocess extracted text.\"\"\"\n",
    "#     # Remove chapter titles and sections\n",
    "#     text = re.sub(r'^(Introduction|Chapter \\d+:|What is|Examples:|Chapter \\d+)', '', text, flags=re.MULTILINE)\n",
    "#     text = re.sub(r'ctitious', 'fictitious', text)\n",
    "#     text = re.sub(r'ISBN[- ]13: \\d{13}', '', text)\n",
    "#     text = re.sub(r'ISBN[- ]10: \\d{10}', '', text)\n",
    "#     text = re.sub(r'Library of Congress Control Number : \\d+', '', text)\n",
    "#     text = re.sub(r'(\\.|\\?|\\!)(\\S)', r'\\1 \\2', text)  # Ensure space after punctuation\n",
    "#     text = re.sub(r'All rights reserved|Copyright \\d{4}', '', text)\n",
    "#     text = re.sub(r'\\n\\s*\\n', '\\n', text)\n",
    "#     text = re.sub(r'[^\\x00-\\x7F]+', ' ', text)\n",
    "#     text = re.sub(r'\\s{2,}', ' ', text)\n",
    "\n",
    "#     # Remove all newlines and replace newlines only after periods\n",
    "#     text = text.replace('\\n', ' ')\n",
    "#     text = re.sub(r'(\\.)(\\s)', r'\\1\\n', text)\n",
    "\n",
    "#     return text\n",
    "\n",
    "# def extract_text_from_pdf(pdf_path):\n",
    "#     \"\"\"Extract text from a PDF file using pdfplumber.\"\"\"\n",
    "#     text = ''\n",
    "#     with pdfplumber.open(pdf_path) as pdf:\n",
    "#         for page in pdf.pages:\n",
    "#             page_text = page.extract_text()\n",
    "#             if page_text:\n",
    "#                 text += page_text + ' '\n",
    "#     return text\n",
    "\n",
    "# def main():\n",
    "#     pdf_path = 'test.pdf'  # Path to your PDF file\n",
    "#     extracted_text = extract_text_from_pdf(pdf_path)\n",
    "#     cleaned_text = clean_extracted_text(extracted_text)\n",
    "\n",
    "#     # Output the cleaned text to a file\n",
    "#     with open('cleaned_text_output.txt', 'w', encoding='utf-8') as file:\n",
    "#         file.write(cleaned_text)\n",
    "\n",
    "\n",
    "# main()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "ename": "ModuleNotFoundError",
     "evalue": "No module named 'fake_useragent'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mModuleNotFoundError\u001b[0m                       Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[18], line 8\u001b[0m\n\u001b[1;32m      6\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mtime\u001b[39;00m\n\u001b[1;32m      7\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mrandom\u001b[39;00m\n\u001b[0;32m----> 8\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mfake_useragent\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m UserAgent\n\u001b[1;32m      9\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mtraceback\u001b[39;00m\n",
      "\u001b[0;31mModuleNotFoundError\u001b[0m: No module named 'fake_useragent'"
     ]
    }
   ],
   "source": [
    "from lxml import etree\n",
    "import csv\n",
    "import requests\n",
    "from email.mime.text import MIMEText\n",
    "import smtplib\n",
    "import time\n",
    "import random\n",
    "from fake_useragent import UserAgent\n",
    "import traceback\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "ename": "AttributeError",
     "evalue": "'NoneType' object has no attribute 'strip'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[39], line 9\u001b[0m\n\u001b[1;32m      7\u001b[0m response \u001b[38;5;241m=\u001b[39m requests\u001b[38;5;241m.\u001b[39mget(url)\n\u001b[1;32m      8\u001b[0m useful_msg  \u001b[38;5;241m=\u001b[39m use_xpath(response, xpath_txt)[\u001b[38;5;241m0\u001b[39m]  \u001b[38;5;66;03m# 获取匹配的第一个元素\u001b[39;00m\n\u001b[0;32m----> 9\u001b[0m title \u001b[38;5;241m=\u001b[39m \u001b[43museful_msg\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;241;43m0\u001b[39;49m\u001b[43m]\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mtext\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mstrip\u001b[49m()\n\u001b[1;32m     11\u001b[0m \u001b[38;5;28mprint\u001b[39m(text_content)\n",
      "\u001b[0;31mAttributeError\u001b[0m: 'NoneType' object has no attribute 'strip'"
     ]
    }
   ],
   "source": [
    "def use_xpath(response, xpath_txt):\n",
    "    html = etree.HTML(response.content.decode(response.apparent_encoding)) # 这里需要解码一下\n",
    "    return html.xpath(xpath_txt)\n",
    "\n",
    "url = 'https://www.scut.edu.cn/new/'  # 替换为你想要爬取的 URL\n",
    "xpath_txt = '/html/body/div[3]/div[1]/div[1]/div/div[1]/div[2]/div/div/div[1]/div[2]'\n",
    "response = requests.get(url)\n",
    "useful_msg  = use_xpath(response, xpath_txt)[0]  # 获取匹配的第一个元素\n",
    "title = useful_msg[0].text.strip()\n",
    "\n",
    "print(text_content)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "base",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.18"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
